package com.miyu.client1.test.magic;

import com.alibaba.fastjson.JSONArray;

import com.alibaba.fastjson.JSONObject;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.JsonFilePageModelPipeline;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class magic implements PageProcessor {
    JSONObject rs = new JSONObject();
    //爬虫配置信息设置
    private Site site = new Site()
            .setCharset("utf-8")    //设置编码
            .setSleepTime(1)        //设置抓取间隔
            .setTimeOut(10000)         //设置超时时间
            .setRetrySleepTime(300)      //设置重试时间
            .setRetryTimes(3);       //设置重试次数


    @Override
    public Site getSite() {
        return this.site;
    }

    public static void main(String[] args) {
        String str ="仙";
       // String startUrl="http://www.147xs.org/search.php?keyword="+str+"";
        String startUrl = "https://m.lingyutxt.com/SearchBook.php?keyword=" + str + "";

        Request request = new Request();
        Map<String,Object> mp = new HashMap<>();
        mp.put("zhangss","ceshishsishisiih");
        request.putExtra("bookRule",mp);
        request.setUrl(startUrl);
       // page.addTargetRequest(request);

        Spider.create(new magic()).addRequest(request)
                .addPipeline(new ConsolePipeline())
                .addPipeline( new JsonFilePipeline("D:\\test\\") )
                // .addPipeline(new com.miyu.client1.test.magic.SavePipeline())
                .thread(50).run();

//        Spider.create(new magic()).addUrl(startUrl)
//                .addPipeline(new ConsolePipeline())
//                .addPipeline( new JsonFilePipeline("D:\\test\\") )
//               // .addPipeline(new com.miyu.client1.test.magic.SavePipeline())
//                .thread(50).run();


    }

    @Override
    public void process(Page page) {

        String url=page.getUrl().get();//得到url，起始为http://www.147xs.org/

        if (url.contains("keyword")){
            List<String> all = page.getHtml().xpath("//div[@class='hot_sale']/a").links().all();
            for (String str: all
                 ) {
                Map<String,Object> bookRule = page.getRequest().getExtras();
                Request request = new Request();
                request.putExtra("bookRule",bookRule);
                request.setUrl(str);
                page.addTargetRequest(request);
              //  page.setSkip(true);
            }

        }else{
            getContent(page);
        }

    }

    private void getContent(Page page) {

        //书名
        String bookName = page.getHtml().xpath("//span[@class='title']/text()").get();
        String bookAuthor = page.getHtml().xpath("//li[@class='author']/a/text()").get();
        String bookKind = page.getHtml().xpath("//li[@class='sort']/text()").get();
        String bookState = page.getHtml().xpath("//div/ul[@id='book_detail']/li[3]/text()").get();
        String bookLastDate = page.getHtml().xpath("//div/ul[@id='book_detail']/li[4]/text()").get();
        String bookHore = page.getHtml().xpath("//div/ul[@id='book_detail']/li[5]/text()").get();
        String bookCover = page.getHtml().xpath("//div[@id='thumb']/img/@src").get();
        String bookDuce = page.getHtml().xpath("//p[@class='review']/text()").get();
        String bookLastChapter = page.getHtml().xpath("//div[@id='chapterlist']/p[1]/a/text()").get();
        String bookLastChapterUrl = page.getHtml().xpath("//div[@id='chapterlist']/p[1]/a/@href").get();
        page.putField("key",site.getDomain().toString());
        page.putField("bookName",bookName);
        page.putField("bookAuthor",bookAuthor);
        page.putField("bookKind",bookKind);
        page.putField("bookState",bookState);
        page.putField("bookLastDate",bookLastDate);
        page.putField("bookHore",bookHore);
        page.putField("bookCover",bookCover);
        page.putField("bookDuce",bookDuce);
        page.putField("bookLastChapter",bookLastChapter);
        page.putField("bookLastChapterUrl",bookLastChapterUrl);


    }

    class SavePipeline implements Pipeline {
        //数据重新封装
        JSONObject rs = new JSONObject();

        @Override
        public void process(ResultItems resultItems, Task task) {

            String key =  resultItems.get("key");
            String bookName = resultItems.get("bookName");

            rs.put(key,resultItems);

            System.out.println(">>>>>>>>>>>>>>>>>"+rs);

        }

    }
}
